from __future__ import annotations

from typing import List, Tuple

from .models import Question


def jaccard_similarity(a: str, b: str) -> float:
    sa = set(a)
    sb = set(b)
    if not sa and not sb:
        return 1.0
    if not sa or not sb:
        return 0.0
    return len(sa & sb) / len(sa | sb)


def deduplicate(questions: List[Question], threshold: float = 0.9) -> Tuple[List[Question], int]:
    kept: List[Question] = []
    removed = 0
    for q in questions:
        duplicate = False
        for k in kept:
            if jaccard_similarity(q.stem, k.stem) >= threshold:
                duplicate = True
                break
        if duplicate:
            removed += 1
        else:
            kept.append(q)
    return kept, removed